

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Download and register a dynamic dataset of (crowd-sourced?) US Marijuana street prices &mdash; geoslurp v1.1.0 documentation</title>
  

  
  
    <link rel="shortcut icon" href="../_static/favicon.ico"/>
  
  
  

  

  
  
    

  

  
  
    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  

  

  
        <link rel="index" title="Index"
              href="../genindex.html"/>
        <link rel="search" title="Search" href="../search.html"/>
    <link rel="top" title="geoslurp v1.1.0 documentation" href="../index.html"/>
        <link rel="up" title="Python Examples" href="pyexamples.html"/>
        <link rel="next" title="Discovery techniques" href="BasicDiscovery.html"/>
        <link rel="prev" title="Downloading and registering “catalogue” datasets" href="UsingCataloguedDsets.html"/> 

  
  <script src="../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

   
  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

          
            <a href="../index.html" class="icon icon-home"> geoslurp
          

          
          </a>

          
            
            
              <div class="version">
                1.1.0
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">Contents:</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../introduction.html">An introduction to Geoslurp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation of the geoslurp package and setting up the PostgreSQL instance</a></li>
<li class="toctree-l1"><a class="reference internal" href="../confsettings.html">Configuration</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../examples.html">Tutorial</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../usingcli.html">Using the command line tool geoslurp</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="pyexamples.html">Python Examples</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="connectnQuery.html">Establish a database connection and retrieve the inventory and User settings</a></li>
<li class="toctree-l3"><a class="reference internal" href="UsingCataloguedDsets.html">Downloading and registering “catalogue” datasets</a></li>
<li class="toctree-l3 current"><a class="current reference internal" href="#">Download and register a dynamic dataset of (crowd-sourced?) US Marijuana street prices</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#Example:-US-street-prices-of-Marijuana">Example: US street prices of Marijuana</a></li>
<li class="toctree-l4"><a class="reference internal" href="#Avoid-reinventing-the-wheel">Avoid reinventing the wheel</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="BasicDiscovery.html">Discovery techniques</a></li>
<li class="toctree-l3"><a class="reference internal" href="export2Portable.html">Exporting parts of the database to portable formats</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../reference.html">Reference Documentation</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">geoslurp</a>
        
      </nav>


      
      <div class="wy-nav-content">
        <div class="rst-content">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> &raquo;</li>
        
          <li><a href="../examples.html">Tutorial</a> &raquo;</li>
        
          <li><a href="pyexamples.html">Python Examples</a> &raquo;</li>
        
      <li>Download and register a dynamic dataset of (crowd-sourced?) US Marijuana street prices</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/examples/CreateDynDSet.ipynb.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  
<style>
/* CSS for nbsphinx extension */

/* remove conflicting styling from Sphinx themes */
div.nbinput,
div.nbinput div.prompt,
div.nbinput div.input_area,
div.nbinput div[class*=highlight],
div.nbinput div[class*=highlight] pre,
div.nboutput,
div.nbinput div.prompt,
div.nbinput div.output_area,
div.nboutput div[class*=highlight],
div.nboutput div[class*=highlight] pre {
    background: none;
    border: none;
    padding: 0 0;
    margin: 0;
    box-shadow: none;
}

/* avoid gaps between output lines */
div.nboutput div[class*=highlight] pre {
    line-height: normal;
}

/* input/output containers */
div.nbinput,
div.nboutput {
    display: -webkit-flex;
    display: flex;
    align-items: flex-start;
    margin: 0;
    width: 100%;
}
@media (max-width: 540px) {
    div.nbinput,
    div.nboutput {
        flex-direction: column;
    }
}

/* input container */
div.nbinput {
    padding-top: 5px;
}

/* last container */
div.nblast {
    padding-bottom: 5px;
}

/* input prompt */
div.nbinput div.prompt pre {
    color: #303F9F;
}

/* output prompt */
div.nboutput div.prompt pre {
    color: #D84315;
}

/* all prompts */
div.nbinput div.prompt,
div.nboutput div.prompt {
    min-width: 8ex;
    padding-top: 0.4em;
    padding-right: 0.4em;
    text-align: right;
    flex: 0;
}
@media (max-width: 540px) {
    div.nbinput div.prompt,
    div.nboutput div.prompt {
        text-align: left;
        padding: 0.4em;
    }
    div.nboutput div.prompt.empty {
        padding: 0;
    }
}

/* disable scrollbars on prompts */
div.nbinput div.prompt pre,
div.nboutput div.prompt pre {
    overflow: hidden;
}

/* input/output area */
div.nbinput div.input_area,
div.nboutput div.output_area {
    padding: 0.4em;
    -webkit-flex: 1;
    flex: 1;
    overflow: auto;
}
@media (max-width: 540px) {
    div.nbinput div.input_area,
    div.nboutput div.output_area {
        width: 100%;
    }
}

/* input area */
div.nbinput div.input_area {
    border: 1px solid #cfcfcf;
    border-radius: 2px;
    background: #f7f7f7;
}

/* override MathJax center alignment in output cells */
div.nboutput div[class*=MathJax] {
    text-align: left !important;
}

/* override sphinx.ext.pngmath center alignment in output cells */
div.nboutput div.math p {
    text-align: left;
}

/* standard error */
div.nboutput div.output_area.stderr {
    background: #fdd;
}

/* ANSI colors */
.ansi-black-fg { color: #3E424D; }
.ansi-black-bg { background-color: #3E424D; }
.ansi-black-intense-fg { color: #282C36; }
.ansi-black-intense-bg { background-color: #282C36; }
.ansi-red-fg { color: #E75C58; }
.ansi-red-bg { background-color: #E75C58; }
.ansi-red-intense-fg { color: #B22B31; }
.ansi-red-intense-bg { background-color: #B22B31; }
.ansi-green-fg { color: #00A250; }
.ansi-green-bg { background-color: #00A250; }
.ansi-green-intense-fg { color: #007427; }
.ansi-green-intense-bg { background-color: #007427; }
.ansi-yellow-fg { color: #DDB62B; }
.ansi-yellow-bg { background-color: #DDB62B; }
.ansi-yellow-intense-fg { color: #B27D12; }
.ansi-yellow-intense-bg { background-color: #B27D12; }
.ansi-blue-fg { color: #208FFB; }
.ansi-blue-bg { background-color: #208FFB; }
.ansi-blue-intense-fg { color: #0065CA; }
.ansi-blue-intense-bg { background-color: #0065CA; }
.ansi-magenta-fg { color: #D160C4; }
.ansi-magenta-bg { background-color: #D160C4; }
.ansi-magenta-intense-fg { color: #A03196; }
.ansi-magenta-intense-bg { background-color: #A03196; }
.ansi-cyan-fg { color: #60C6C8; }
.ansi-cyan-bg { background-color: #60C6C8; }
.ansi-cyan-intense-fg { color: #258F8F; }
.ansi-cyan-intense-bg { background-color: #258F8F; }
.ansi-white-fg { color: #C5C1B4; }
.ansi-white-bg { background-color: #C5C1B4; }
.ansi-white-intense-fg { color: #A1A6B2; }
.ansi-white-intense-bg { background-color: #A1A6B2; }

.ansi-default-inverse-fg { color: #FFFFFF; }
.ansi-default-inverse-bg { background-color: #000000; }

.ansi-bold { font-weight: bold; }
.ansi-underline { text-decoration: underline; }

/* CSS overrides for sphinx_rtd_theme */

/* 24px margin */
.nbinput.nblast,
.nboutput.nblast {
    margin-bottom: 19px;  /* padding has already 5px */
}

/* ... except between code cells! */
.nblast + .nbinput {
    margin-top: -19px;
}

.admonition > p:before {
    margin-right: 4px;  /* make room for the exclamation icon */
}
</style>
<div class="section" id="Download-and-register-a-dynamic-dataset-of-(crowd-sourced?)-US-Marijuana-street-prices">
<h1>Download and register a dynamic dataset of (crowd-sourced?) US Marijuana street prices<a class="headerlink" href="#Download-and-register-a-dynamic-dataset-of-(crowd-sourced?)-US-Marijuana-street-prices" title="Permalink to this headline">¶</a></h1>
<p>To download and register a dataset, one can create a new class which is
inherited from
<a class="reference external" href="../reference/geoslurp.dataset.html#geoslurp.dataset.dataSetBase.DataSet">DataSet</a>.
Two function <em>pull</em> and <em>register</em> can then be implemented so the data
can be downloaded and registered in the database.</p>
<div class="section" id="Example:-US-street-prices-of-Marijuana">
<h2>Example: US street prices of Marijuana<a class="headerlink" href="#Example:-US-street-prices-of-Marijuana" title="Permalink to this headline">¶</a></h2>
<p>I found this curious dataset (csv format) on this
<a class="reference external" href="http://blog.yhat.com/posts/7-funny-datasets.html">blog</a>. So let’s
use it as an example. It’s unclear where it get’s the data from but I
suspect it has been scraped from
<a class="reference external" href="http://www.priceofweed.com/directory">here</a> The accuracy is
therefore questionable but it serves it’s purpose here nevertheless.</p>
<div class="section" id="Create-the-stucture-of-the-database-table-using-sqalchemy-ORM-methods">
<h3>Create the stucture of the database table using sqalchemy ORM methods<a class="headerlink" href="#Create-the-stucture-of-the-database-table-using-sqalchemy-ORM-methods" title="Permalink to this headline">¶</a></h3>
<p>Since we know how are table is going to look like we can set it up
according to <a class="reference external" href="https://docs.sqlalchemy.org/en/13/orm/tutorial.html#declare-a-mapping">SQAlchemy’s
tutorial</a></p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [1]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">sqlalchemy</span> <span class="k">import</span> <span class="n">MetaData</span><span class="p">,</span><span class="n">Column</span><span class="p">,</span><span class="n">Float</span><span class="p">,</span><span class="n">Integer</span><span class="p">,</span><span class="n">String</span>
<span class="kn">from</span> <span class="nn">sqlalchemy.dialects.postgresql</span> <span class="k">import</span> <span class="n">TIMESTAMP</span>
<span class="kn">from</span> <span class="nn">sqlalchemy.ext.declarative</span> <span class="k">import</span> <span class="n">declarative_base</span>

<span class="n">scheme</span><span class="o">=</span><span class="s2">&quot;public&quot;</span>

<span class="c1"># Setup the postgres table using methods as specified with sqlalchemy</span>
<span class="n">WeedTBase</span><span class="o">=</span><span class="n">declarative_base</span><span class="p">(</span><span class="n">metadata</span><span class="o">=</span><span class="n">MetaData</span><span class="p">(</span><span class="n">schema</span><span class="o">=</span><span class="n">scheme</span><span class="p">))</span>

<span class="k">class</span> <span class="nc">Weedtable</span><span class="p">(</span><span class="n">WeedTBase</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Defines the Marijuana PostgreSQL table&quot;&quot;&quot;</span>
    <span class="n">__tablename__</span><span class="o">=</span><span class="s1">&#39;usweedprices&#39;</span>
    <span class="nb">id</span><span class="o">=</span><span class="n">Column</span><span class="p">(</span><span class="n">Integer</span><span class="p">,</span><span class="n">primary_key</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
    <span class="n">state</span><span class="o">=</span><span class="n">Column</span><span class="p">(</span><span class="n">String</span><span class="p">)</span>
    <span class="n">highq</span><span class="o">=</span><span class="n">Column</span><span class="p">(</span><span class="n">Float</span><span class="p">)</span>
    <span class="n">medq</span><span class="o">=</span><span class="n">Column</span><span class="p">(</span><span class="n">Float</span><span class="p">)</span>
    <span class="n">lowq</span><span class="o">=</span><span class="n">Column</span><span class="p">(</span><span class="n">Float</span><span class="p">)</span>
    <span class="n">time</span><span class="o">=</span><span class="n">Column</span><span class="p">(</span><span class="n">TIMESTAMP</span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="Create-a-helper-function-to-extract-relevant-information-from-the-file">
<h3>Create a helper function to extract relevant information from the file<a class="headerlink" href="#Create-a-helper-function-to-extract-relevant-information-from-the-file" title="Permalink to this headline">¶</a></h3>
<p>This will return dictionaries with keys who match the corresponding
column names above</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [2]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span>
<span class="k">def</span> <span class="nf">metaExtract</span><span class="p">(</span><span class="n">csvfile</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;A little generator which extracts rows from &quot;&quot;&quot;</span>
    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">csvfile</span><span class="p">,</span><span class="s1">&#39;rt&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fid</span><span class="p">:</span>
        <span class="n">ln</span><span class="o">=</span><span class="n">fid</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;,&quot;</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">ln</span> <span class="ow">in</span> <span class="n">fid</span><span class="p">:</span>
            <span class="n">lnspl</span><span class="o">=</span><span class="n">ln</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;,&quot;</span><span class="p">)</span>
            <span class="n">dt</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">lnspl</span><span class="p">[</span><span class="mi">7</span><span class="p">],</span> <span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> &quot;</span><span class="p">)</span>
            <span class="k">try</span><span class="p">:</span>
                <span class="n">meta</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;state&quot;</span><span class="p">:</span><span class="n">lnspl</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span><span class="s2">&quot;highq&quot;</span><span class="p">:</span><span class="nb">float</span><span class="p">(</span><span class="n">lnspl</span><span class="p">[</span><span class="mi">1</span><span class="p">]),</span><span class="s2">&quot;medq&quot;</span><span class="p">:</span><span class="nb">float</span><span class="p">(</span><span class="n">lnspl</span><span class="p">[</span><span class="mi">3</span><span class="p">]),</span><span class="s2">&quot;lowq&quot;</span><span class="p">:</span><span class="nb">float</span><span class="p">(</span><span class="n">lnspl</span><span class="p">[</span><span class="mi">5</span><span class="p">]),</span><span class="s2">&quot;time&quot;</span><span class="p">:</span><span class="n">dt</span><span class="p">}</span>
            <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
                <span class="c1">#skip entries with NA values</span>
                <span class="k">continue</span>
            <span class="k">yield</span> <span class="n">meta</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="Inherit-from-DataSet">
<h3>Inherit from <a class="reference external" href="../reference/geoslurp.dataset.html#geoslurp.dataset.dataSetBase.DataSet">DataSet</a><a class="headerlink" href="#Inherit-from-DataSet" title="Permalink to this headline">¶</a></h3>
<p>In this case, we implement the <em>pull</em> method (where to download the
data). and the <em>register</em> method. Note that we explicitly insert the
sqlalchemy table as a member in the class, and specify the scheme name
as a member.</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [3]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">geoslurp.dataset</span> <span class="k">import</span> <span class="n">DataSet</span>
<span class="kn">from</span> <span class="nn">geoslurp.datapull.http</span> <span class="k">import</span> <span class="n">Uri</span> <span class="k">as</span> <span class="n">http</span>
<span class="kn">import</span> <span class="nn">os</span>

<span class="k">class</span> <span class="nc">USWeedPrices</span><span class="p">(</span><span class="n">DataSet</span><span class="p">):</span>
    <span class="n">scheme</span><span class="o">=</span><span class="n">scheme</span>
    <span class="n">csvfile</span><span class="o">=</span><span class="s2">&quot;marijuana-street-price-clean.csv&quot;</span>
    <span class="n">table</span><span class="o">=</span><span class="n">Weedtable</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span><span class="n">dbcon</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">dbcon</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">pull</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Pulls the csv file from the interwebs&quot;&quot;&quot;</span>
        <span class="n">weedurl</span><span class="o">=</span><span class="n">http</span><span class="p">(</span><span class="s2">&quot;http://blog.yhat.com/static/misc/data/marijuana-street-price-clean.csv&quot;</span><span class="p">)</span>
        <span class="n">uri</span><span class="p">,</span><span class="n">updated</span><span class="o">=</span><span class="n">weedurl</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cacheDir</span><span class="p">())</span>

    <span class="k">def</span> <span class="nf">register</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">truncateTable</span><span class="p">()</span>

        <span class="c1">#insert in bulk mode</span>
<span class="c1">#         metalist=[meta for meta in metaExtract(os.path.join(self.cacheDir(),self.csvfile))]</span>
<span class="c1">#         self.bulkInsert(metalist)</span>

        <span class="c1">#insert by entry</span>
        <span class="k">for</span> <span class="n">meta</span> <span class="ow">in</span> <span class="n">metaExtract</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cacheDir</span><span class="p">(),</span><span class="bp">self</span><span class="o">.</span><span class="n">csvfile</span><span class="p">)):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">addEntry</span><span class="p">(</span><span class="n">meta</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">updateInvent</span><span class="p">()</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="Create-an-instance-of-the-class-and-call-the-pull,-and-register-methods-to-download-and-register-the-data">
<h3>Create an instance of the class and call the pull, and register methods to download and register the data<a class="headerlink" href="#Create-an-instance-of-the-class-and-call-the-pull,-and-register-methods-to-download-and-register-the-data" title="Permalink to this headline">¶</a></h3>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [4]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">geoslurp.config</span> <span class="k">import</span> <span class="n">setInfoLevel</span>
<span class="kn">from</span> <span class="nn">geoslurp.db</span> <span class="k">import</span> <span class="n">geoslurpConnect</span>
<span class="n">setInfoLevel</span><span class="p">()</span>


<span class="n">gpcon</span><span class="o">=</span><span class="n">geoslurpConnect</span><span class="p">(</span><span class="n">readonlyuser</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>


<span class="n">usWeed</span><span class="o">=</span><span class="n">USWeedPrices</span><span class="p">(</span><span class="n">gpcon</span><span class="p">)</span>

<span class="n">usWeed</span><span class="o">.</span><span class="n">pull</span><span class="p">()</span>
<span class="n">usWeed</span><span class="o">.</span><span class="n">register</span><span class="p">()</span>

</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="stderr output_area docutils container">
<div class="highlight"><pre>
Geoslurp-INFO: Downloading /tmp/geoslurp/public/usweedprices/marijuana-street-price-clean.csv
</pre></div></div>
</div>
</div>
</div>
<div class="section" id="Avoid-reinventing-the-wheel">
<h2>Avoid reinventing the wheel<a class="headerlink" href="#Avoid-reinventing-the-wheel" title="Permalink to this headline">¶</a></h2>
<p>One can imagine that having a csv file is pretty common. The above could
therefore also be simplified by inheriting from
<a class="reference external" href="../reference/geoslurp.dataset.html#geoslurp.dataset.dataSetBase.DataSet">PandasBase</a>,
which is demonstrated bellow. Note that in this case, the register
function of <strong>PandasBase</strong> works as is, and there is no need to
reimplement it.</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [5]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre>
<span></span><span class="c1">#get rid of the previous dataset</span>
<span class="n">usWeed</span><span class="o">.</span><span class="n">purgeentry</span><span class="p">()</span> <span class="c1">#entry in the database</span>
<span class="n">usWeed</span><span class="o">.</span><span class="n">purgecache</span><span class="p">()</span> <span class="c1"># the cached data</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="stderr output_area docutils container">
<div class="highlight"><pre>
Geoslurp-INFO: Deleting usweedprices entry
Geoslurp-INFO: Pruning directory /tmp/geoslurp/public/usweedprices
</pre></div></div>
</div>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [6]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">geoslurp.dataset</span> <span class="k">import</span> <span class="n">PandasBase</span>

<span class="k">class</span>  <span class="nc">USWeedPrices2</span><span class="p">(</span><span class="n">PandasBase</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span><span class="n">dbconn</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">dbconn</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">pdfile</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cacheDir</span><span class="p">(),</span><span class="s2">&quot;marijuana-street-price-clean.csv&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">pull</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;same as above but we now also only download when newer than specified&quot;&quot;&quot;</span>
        <span class="n">weedurl</span><span class="o">=</span><span class="n">http</span><span class="p">(</span><span class="s2">&quot;http://blog.yhat.com/static/misc/data/marijuana-street-price-clean.csv&quot;</span><span class="p">,</span><span class="n">lastmod</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2018</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">))</span>
        <span class="n">uri</span><span class="p">,</span><span class="n">updated</span><span class="o">=</span><span class="n">weedurl</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cacheDir</span><span class="p">(),</span><span class="n">check</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>




</pre></div>
</div>
</div>
<p>Once the class is defined, one can again pull the data and call the
register function. Examples for querying this dataset are found in the
example notebook on discovery.</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre>
<span></span>In [7]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre>
<span></span><span class="n">usWeed2</span><span class="o">=</span><span class="n">USWeedPrices2</span><span class="p">(</span><span class="n">gpcon</span><span class="p">)</span>

<span class="n">usWeed2</span><span class="o">.</span><span class="n">pull</span><span class="p">()</span>

<span class="n">usWeed2</span><span class="o">.</span><span class="n">register</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="stderr output_area docutils container">
<div class="highlight"><pre>
Geoslurp-INFO: Already Downloaded, skipping /tmp/geoslurp/public/usweedprices2/marijuana-street-price-clean.csv
Geoslurp-INFO: Filling pandas table public.usweedprices2 with data from /tmp/geoslurp/public/usweedprices2/marijuana-street-price-clean.csv
</pre></div></div>
</div>
</div>
</div>


           </div>
           <div class="articleComments">
            
           </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="BasicDiscovery.html" class="btn btn-neutral float-right" title="Discovery techniques" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="UsingCataloguedDsets.html" class="btn btn-neutral" title="Downloading and registering “catalogue” datasets" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2020, Roelof Rietbroek.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../',
            VERSION:'v1.1.0',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: '.txt'
        };
    </script>
      <script type="text/javascript" src="../_static/jquery.js"></script>
      <script type="text/javascript" src="../_static/underscore.js"></script>
      <script type="text/javascript" src="../_static/doctools.js"></script>

  

  
  
    <script type="text/javascript" src="../_static/js/theme.js"></script>
  

  
  
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>